import scrapy
from testPro.items import tcntItem, tcntQuota, tcntQuotaMap
import testPro.utils.RegularExpression as RE
import json

class TnctSpider(scrapy.Spider):
    name = 'tcnt-family'
    allowed_domains = ['cloud.tencent.com']

    def __init__(self, category=None, *args, **kwargs):
        super(TnctSpider, self).__init__(*args, **kwargs)
        with open("./testPro/conf/config.json",'r') as load_f1:
            self.config = json.load(load_f1)
        with open("./testPro/conf/data.json",'r') as load_f2:
            self.table = json.load(load_f2)
        self.start_urls = self.config[self.name]["urlList"]

    def caseCPU(self, val):
        return ("处理器" in val or "CPU" in val) and "配比" not in val
    def caseProp(self, val):
        return  "配比" in val
    def caseDisk(self, val):
        return  "硬盘" in val
    def caseInsc(self, val):
        return  "指令集" in val
    def caseNet(self, val):
        return  ("网络" in val and "带宽" in val)

    # 解释器
    def parse(self, response):
        if response.url.split('/')[-1] == "19700" or response.url.split('/')[-1] == "63854":
            h3 = response.xpath("//div[@class=\"markdown-text-box\"]/h3")
            for head3 in h3:
                item = tcntItem()
                item["Family"] = RE.find_unchinese(head3.xpath("text()").extract()[0])
                item["CPU"] = '-'
                item["GPU"] = '-'
                item["ClockSpeed"] = '-'
                item["Proportion"] = '-'
                item["Usescenes"] = head3.xpath("./following-sibling::ul[1]/text()").extract()[0]
                if "GN7" in item["Family"] or "GN6" in item["Family"]:
                    hardware = head3.xpath("./following-sibling::ul[3]/li")
                else:
                    hardware = head3.xpath("./following-sibling::ul[2]/li")
                for i,line in enumerate(hardware):
                    if "CPU" in line.xpath("./strong/text()").extract()[0]:
                        item["CPU"] = RE.find_unchinese(line.xpath("string(.)").extract()[0])
                        item["ClockSpeed"] = RE.fing_ClockSpeed(line.xpath("string(.)").extract()[0])
                    elif "GPU" in line.xpath("./strong/text()").extract()[0]:
                        item["GPU"] = line.xpath("string(.)").extract()[0]
                yield item
        elif response.url.split('/')[-1] == "63404":
            div = response.xpath("//div[@class='DX-accordion rno-quesion-items']")
            sec = div.xpath("./div[@class='DX-accordion-item rno-quesion-item']")\
                +div.xpath("./div[@class='DX-accordion-item rno-quesion-item active']")
            for instance in sec:
                item = tcntItem()
                item["Family"] = RE.find_unchinese(instance.xpath("./div[1]/h3/text()").extract()[0])
                item["CPU"] = '-'
                item["GPU"] = '-'
                item["ClockSpeed"] = '-'
                item["Proportion"] = '-'
                item["Usescenes"] = '-'
                instance = instance.xpath("./div/div")
                Usescenes = instance.xpath("./ul[1]")
                item["Usescenes"] = Usescenes.xpath("string(.)").extract()[0]

                if "BMG5t" in item["Family"] or "BMG5v" in item["Family"] or "BMGNV4" in item["Family"]:
                    hardware = head3.xpath("./ul[3]/li")
                else:
                    hardware = instance.xpath("./ul[2]/li")
                for i,line in enumerate(hardware):
                    if "CPU" in line.xpath("string(.)").extract()[0]:
                        item["CPU"] = RE.find_unchinese(line.xpath("string(.)").extract()[0])
                        item["ClockSpeed"] = RE.fing_ClockSpeed(line.xpath("string(.)").extract()[0])
                
                # 1.CPU 内存信息
            
                # item["CPU"] = RE.find_target(temp,self.config[self.name]["RE-CPU"],True)
                item['CPU'] = RE.del_target(item['CPU'],r'-?\d+\.?\d*e?-?\d*GHz?')
                item['CPU'] = RE.del_target(item['CPU'],r'，')
                item['CPU'] = RE.del_target(item['CPU'],r'、')

                # 2.主频

                CpuMapKey = self.table["CPUMap"].keys()
                for i in CpuMapKey:
                    if i in item["CPU"]:
                        item["CPU"] = self.table["CPUMap"][i] + ' ' +  RE.find_CPUnum(item["CPU"])
                
                yield item
        else:
            # 这一行有非常明确的意义，sect2 就是family的
            for sect2 in response.xpath(self.config[self.name]["XPATH-section2"]): 
                # 每个sect2 对应一个family
                # 接下来对sect2 下的 sect3 进行分析
                # sect2 包含一个 h3
                # sect2 包含一个 paragraph, 对实例进行基本介绍
                # sect2 包含四个 sect3 个部分: 实例特点、使用场景、实例要求、回到顶部
                item = tcntItem()
                item["Family"] = '-'
                item["CPU"] = '-'
                item["GPU"] = '-'
                item["ClockSpeed"] = '-'
                item["Proportion"] = '-'
                item["Usescenes"] = '-'

                item["Family"] = RE.find_unchinese(sect2.xpath(self.config[self.name]["XPATH-family"]).extract()[0]).split(' ')[-1]

                for sect3 in sect2.xpath(self.config[self.name]["XPATH-section3"]):
                    sect3Title = sect3.xpath('h4').xpath('string(.)').extract()[0]

                    if sect3Title == self.config[self.name]["RE-feature"]:
                        CPUDisc = ''
                        PropDisc = ''
                        DiskDisc = ''
                        InscDisc = ''
                        NetDisc = ''
                        for li in sect3.xpath('.//li'):
                            liString = li.xpath('string(.)').extract()[0]
                            if self.caseCPU(liString):
                                CPUDisc += RE.find_unchinese(liString).replace(' ','')
                            elif self.caseProp(liString):
                                PropDisc += RE.find_unchinese(liString).replace(' ','')
                            elif self.caseDisk(liString):
                                DiskDisc += RE.find_unchinese(liString).replace(' ','')
                            elif self.caseInsc(liString):
                                InscDisc += RE.find_unchinese(liString).replace(' ','')
                            elif self.caseNet(liString):
                                NetDisc += RE.find_unchinese(liString).replace(' ','')
                        
                        if item["Family"] in self.table[self.name].keys():
                            item["CPU"] = self.table[self.name][item["Family"]]["CPU"]
                            # print(self.table[self.name]["data"][item["Family"]])
                            item["ClockSpeed"] = self.table[self.name][item["Family"]]["ClockSpeed"]
                        else:
                            item['CPU'] = RE.emplace_enter(CPUDisc)
                            item['ClockSpeed'] = RE.fing_ClockSpeed(item['CPU'])
                            item['CPU'] = RE.del_target(item['CPU'],r'-?\d+\.?\d*e?-?\d*GHz?')
                            item['CPU'] = '-' if item['CPU'].strip()=='' else item['CPU'].strip()
                        item['Proportion'] = RE.emplace_enter(PropDisc)
                        # item['ISA'] = RE.emplace_enter(InscDisc)
                        # item['Network'] = RE.emplace_enter(NetDisc)
                        # item['Disk'] = RE.emplace_enter(DiskDisc)

                    elif sect3Title == self.config[self.name]["RE-usescenes"]:
                        USCS = ''
                        for li in sect3.xpath('.//li'):
                            liString = li.xpath('string(.)').extract()[0]
                            USCS += liString.replace(' ','')
                        item['Usescenes'] = '-' if RE.emplace_enter(USCS).strip()=='' else RE.emplace_enter(USCS).strip()
                    elif sect3Title == self.config[self.name]["RE-takes"]:
                        pass
                    else:
                        pass
                yield item

class TnctQuotaSpider(scrapy.Spider):
    name = 'tcnt-family-details'
    allowed_domains = ['cloud.tencent.com']

    def __init__(self, category=None, *args, **kwargs):
        super(TnctQuotaSpider, self).__init__(*args, **kwargs)
        with open("./testPro/conf/config.json",'r') as load_f1:
            self.config = json.load(load_f1)
        with open("./testPro/conf/data.json",'r') as load_f2:
            self.table = json.load(load_f2)
        self.start_urls = self.config[self.name]["urlList"]

    # 解释器
    def parse(self, response):

        # 这一行有非常明确的意义，sect2 就是family的
        sects = response.xpath(self.config[self.name]["XPATH-section1"])[3:] 
        tables = sects.xpath(self.config[self.name]["XPATH-family-tables"])
        for table in tables:
            thead = []
            
            for i,line in enumerate(table.xpath(self.config[self.name]["XPATH-tbody-lines"])):
                if i == 0:
                    thead = line.xpath(".//p/text()").extract()
                else:
                    item = tcntQuota()
                    # 初始化
                    for value in tcntQuotaMap.values():
                        item[value] = '-'
                    for j,content in enumerate(line.xpath(".//p/text()").extract()):
                        item[tcntQuotaMap[thead[j]]] = RE.find_details_translator(content)
                    yield item


                    
